import urllib2
from baike_spider import redis_url_manager

class HtmlDownloader(object):
    def download(self, url):
        if url is None:
            return None
        # response = urllib2.urlopen(url)
        request =urllib2.Request(url)
        request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
        response = urllib2.urlopen(request)
        if response.getcode() != 200:
            redis_url_manager.RedisManager().del_url(url)
            return None
        return response.read()
